package common.main.google;

import common.http.HtmlInfo;
import common.http.SimpleHttp;
import common.system.FileOperation;
import common.utils.DomTree;
import common.utils.MD5Util;
import org.apache.log4j.Logger;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.NodeList;

import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

public class ScholarAuthorInfoStartMain {

    public static Logger logger = Logger.getLogger(ScholarAuthorInfoStartMain.class);

    public static SimpleHttp http = new SimpleHttp();
    public static HtmlInfo html = new HtmlInfo();

    public static void main(String[] args) throws InterruptedException, SQLException {
        //配置数据库等
        Config.init();

        html.setEncode("utf-8");
        //String ua = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 QIHU 360SE";

        String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";
        html.setUa(ua);
        //设置代理。
        html.setProxy(Config.httpProxy);
        //入口url模板
        final String entrance_url = "https://scholar.google.com.hk/scholar?hl=zh-CN&as_sdt=0%2C5&q=<keyword>";

        String content = FileOperation.read("config/getAuthorInfoByArticleTitle.txt");

        String []lines = content.split("\n");

        String line = null;

        String allMD5 = SaveDataToSql.getAllAuthorInfoMD5();

        for(int i = 0;i < lines.length;i++){
            line = lines[i];

            String title = line.trim();

            if(title.startsWith("#") || title.length() < 1){
                continue;
            }

            String url = entrance_url.replace("<keyword>",
                    title.replace("  "," ").replace(" ","+").replace(">",""));

            logger.info("searchKeyword: " + title + "\n url: " + url);

            html.setOrignUrl(url);
            try{
                //请求url
                http.simpleGet(html);
            }catch (IllegalArgumentException e){
                logger.info("searchKeyword: " + title +" 含有特殊字符,跳过");
                FileOperation.appendWrite(title + "\n","config/notExecute.txt");
                Thread.sleep(15 * 1000);
                continue;
            }catch (NotFoundException e){
                logger.info(e.getMessage() + "   1分钟后重试");
                Thread.sleep(60 * 1000);
                i--;
                continue;
            }
            String htmlContent = html.getContent();


            if(htmlContent==null){
                logger.warn("获取列表页数据失败，请检查代理是否正确，网络是否连通，15秒后自动重试");
                Thread.sleep(15 * 1000);
                i--;
                continue;

            } else if(htmlContent.contains("与所有文章均不相符")){
                logger.info("该关键词查询不到有关文章信息，跳过，等待15s执行下一个关键词");
                FileOperation.appendWrite(title + "\n","config/notExecute.txt");
                Thread.sleep(15 * 1000);
                continue;
            }

            DocumentFragment node = DomTree.getNode(htmlContent,html.getEncode());

            logger.info("获取列表页数据成功，等待15s");
            Thread.sleep(15 * 1000);

            List<AuthorInfo> list = extractAuthors(node);

            if(list != null && list.size() != 0){

                for(int j = 0 ;j < list.size();j++){
                    AuthorInfo authorInfo = list.get(j);
                    authorInfo.setSearchKeyword(title);
                    authorInfo.setMd5(MD5Util.MD5(authorInfo.getAuthorName() + authorInfo.getSearchKeyword()));
                    if(allMD5.contains(authorInfo.getMd5())){
                        list.remove(j);
                        j--;
                    }
                }
                logger.info("保存作者信息到数据库,list size:" + list.size());
                SaveDataToSql.insertAuthorInfo(list);
            }else if(list == null){
                logger.info("人机验证时等待10分钟");
                //人机验证时等待10分钟
                Thread.sleep(10 * 60 * 1000);
                i--;
                continue;
            }
            logger.info("获取完一个关键词信息，等待20s");
            Thread.sleep(20 * 1000);
        }

        logger.info("所有任务执行完成");
        System.exit(0);
    }

    public static List<AuthorInfo> extractAuthors(DocumentFragment node) throws InterruptedException {
        logger.info("开始获取作者信息");
        NodeList paperInfoNodeList = DomTree.commonList(GoogleScholarXpath.paperInfo,node);
        if(paperInfoNodeList.getLength() > 0){
            //  获取文章标题
            NodeList articleTitleNodeList = DomTree.commonList(GoogleScholarXpath.title,node);
            String articleTitle = null;
            if(articleTitleNodeList.getLength() > 0){
                articleTitle = articleTitleNodeList.item(0).getTextContent();
            }
            //获取文章url
            NodeList articleUrlNodeList = DomTree.commonList(GoogleScholarXpath.url.replace("<n>","1"),node);
            String articleUrl = null;
            if(articleUrlNodeList.getLength() > 0){
                articleUrl = articleUrlNodeList.item(0).getTextContent();
            }

            //获取作者名字以及url
            NodeList authorNodeList = DomTree.commonList(GoogleScholarXpath.AuthorInfo_Name,node);
            NodeList authorUrlNodeList = DomTree.commonList(GoogleScholarXpath.AuthorInfo_url,node);

            List<AuthorInfo> list = new ArrayList<>();

            if(authorNodeList.getLength() > 0){
                String allAuthorName = authorNodeList.item(0).getTextContent();
                if(allAuthorName.contains("-")){
                    allAuthorName = allAuthorName.split("-")[0];
                }
                String []authorNameList = allAuthorName.split(",");
                for(int i = 0;i < authorNameList.length;i++){
                    AuthorInfo authorInfo = new AuthorInfo();
                    String authorName = authorNameList[i].trim().replaceAll(" ","");
                    logger.info("获取作者: " + authorName +" 的信息");
                    String authorUrl = "";
                    for(int k = 0;k < authorUrlNodeList.getLength();k++){
                        String authorNameTmp = authorUrlNodeList.item(k).getTextContent().trim();
                        if(authorName.equals(authorNameTmp)){
                            authorUrl = "scholar.google.com.hk" + authorUrlNodeList.item(k).getAttributes().getNamedItem("href").getTextContent();
                            extractAuthorInfo(authorInfo,authorUrl);
                            logger.info("获取完一个作者信息，等待20s");
                            Thread.sleep(20 * 1000);
                            break;
                        }
                    }
                    authorInfo.setAuthorName(authorName);
                    authorInfo.setAuthorUrl(authorUrl);
                    authorInfo.setAuthorSort(i + 1);
                    authorInfo.setArticleUrl(articleUrl);
                    authorInfo.setArticleName(articleTitle);

                    list.add(authorInfo);
                }
            }
            return list;
        }

        return null;
    }
    public static void extractAuthorInfo(AuthorInfo authorInfo,String authorUrl) throws InterruptedException {

        html.setOrignUrl(authorUrl);
        http.simpleGet(html);

        if(html.getContent() == null){
            logger.warn("获取列表页数据失败，请检查代理是否正确，网络是否连通，15秒后自动重试");
            Thread.sleep(15 * 1000);
            extractAuthorInfo(authorInfo,authorUrl);
            return;
        }

        DocumentFragment node = DomTree.getNode(html.getContent(),html.getEncode());

        //获取机构信息
        NodeList affNodeList = DomTree.commonList(GoogleScholarXpath.AuthorInfo_affiliation,node);
        String affName = "";
        for(int i = 0;i < affNodeList.getLength();i++){
            affName = affName + affNodeList.item(i).getTextContent() + ",";
        }
        authorInfo.setAffiliation(affName);
        //获取hIndex
        NodeList hIndexNodeList = DomTree.commonList(GoogleScholarXpath.AuthorInfo_hIndex,node);
        if(hIndexNodeList.getLength() > 0){
            authorInfo.sethIndex(Integer.valueOf(hIndexNodeList.item(0).getTextContent()));
        }

        //获取作者历年被引数
        NodeList yearNodeList = DomTree.commonList(GoogleScholarXpath.AuthorInfo_years_cite_year,node);
        NodeList citeNodeList = DomTree.commonList(GoogleScholarXpath.AuthorInfo_years_cite_cite,node);
        String yearsCite = "";
        for(int i = 0;i < yearNodeList.getLength();i++){
            yearsCite = yearsCite + yearNodeList.item(i).getTextContent() + "=" ;
            for(int j = 0;j < citeNodeList.getLength();j++){
                String style = citeNodeList.item(j).getAttributes().getNamedItem("style").getTextContent();
                int index = Integer.valueOf(style.split("z-index:")[1]);
                if(index == yearNodeList.getLength() - i){
                    yearsCite = yearsCite + citeNodeList.item(j).getFirstChild().getTextContent()+";";
                }
            }
            if(yearsCite.endsWith("=")){
                yearsCite = yearsCite + "0;";
            }
        }
        authorInfo.setAuthorYearsCite(yearsCite);

        //判断是否有查看全部合作作者的按钮，有的话请求查看全部的url，没有的话直接获取合作作者的信息
        NodeList findAllNodeList = DomTree.commonList(GoogleScholarXpath.AuthorInfo_coAuthor_findAll,node);
        StringBuilder authorNameAff = new StringBuilder();
        if(findAllNodeList.getLength() > 0){
            String user = authorUrl.split("user=")[1].split("&")[0];
            String coAuthorUrl = "scholar.google.com.hk/citations?view_op=list_colleagues&hl=zh-CN&json=&user=<user>";

            html.setOrignUrl(coAuthorUrl.replace("<user>",user));
            http.simpleGet(html);
            DocumentFragment coAuthorNode = DomTree.getNode(html.getContent(),html.getEncode());
            //获取合作作者的名字
            NodeList coAuthorNameNodeListFirst = DomTree.commonList(GoogleScholarXpath.AuthorInfo_coAuthor_name_first,coAuthorNode);
            for(int i = 0;i < coAuthorNameNodeListFirst.getLength();i++){
                NodeList coAuthorAffNodeList = DomTree.commonList(GoogleScholarXpath.AuthorInfo_coAuthor_affiliation_first.replace("<n>",i + 1 + ""),coAuthorNode);
                authorNameAff.append(coAuthorNameNodeListFirst.item(i).getTextContent() + ":");
                for(int j = 0;j < coAuthorAffNodeList.getLength();j++){
                    authorNameAff.append(coAuthorAffNodeList.item(j).getTextContent() + ";");
                    break;
                }
            }

        }else{
            NodeList coAuthorNameNodeListSec = DomTree.commonList(GoogleScholarXpath.AuthorInfo_coAuthor_name_second,node);
            for(int i = 0;i < coAuthorNameNodeListSec.getLength();i++){
                NodeList coAuthorAffNodeList = DomTree.commonList(GoogleScholarXpath.AuthorInfo_coAuthor_affiliation_second.replace("<n>",i + 1 + ""),node);
                authorNameAff.append(coAuthorNameNodeListSec.item(i).getTextContent() + ":");
                for(int j = 0;j < coAuthorAffNodeList.getLength();j++){
                    authorNameAff.append(coAuthorAffNodeList.item(j).getTextContent() + ";");
                    break;
                }
            }
        }
        authorInfo.setCoAuthors(authorNameAff.toString());
    }
}
